{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "11a1f337",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import time\n",
    "\n",
    "import requests\n",
    "\n",
    "import urllib3 # 处理http请求\n",
    "urllib3.disable_warnings()\n",
    "\n",
    "# 进度条库\n",
    "from tqdm import tqdm\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9531bd96",
   "metadata": {},
   "outputs": [],
   "source": [
    "cookies = {\n",
    "'BDqhfp': '%E7%8B%97%E7%8B%97%26%26NaN-1undefined%26%2618880%26%2621',\n",
    "'BIDUPSID': '06338E0BE23C6ADB52165ACEB972355B',\n",
    "'PSTM': '1646905430',\n",
    "'BAIDUID': '104BD58A7C408DABABCAC9E0A1B184B4:FG=1',\n",
    "'BDORZ': 'B490B5EBF6F3CD402E515D22BCDA1598',\n",
    "'H_PS_PSSID': '35836_35105_31254_36024_36005_34584_36142_36120_36032_35993_35984_35319_26350_35723_22160_36061',\n",
    "'BDSFRCVID': '8--OJexroG0xMovDbuOS5T78igKKHJQTDYLtOwXPsp3LGJLVgaSTEG0PtjcEHMA-2ZlgogKK02OTH6KF_2uxOjjg8UtVJeC6EG0Ptf8g0M5',\n",
    "'H_BDCLCKID_SF': 'tJPqoKtbtDI3fP36qR3KhPt8Kpby2D62aKDs2nopBhcqEIL4QTQM5p5yQ2c7LUvtynT2KJnz3Po8MUbSj4QoDjFjXJ7RJRJbK6vwKJ5s5h5nhMJSb67JDMP0-4F8exry523ioIovQpn0MhQ3DRoWXPIqbN7P-p5Z5mAqKl0MLPbtbb0xXj_0D6bBjHujtT_s2TTKLPK8fCnBDP59MDTjhPrMypomWMT-0bFH_-5L-l5js56SbU5hW5LSQxQ3QhLDQNn7_JjOX-0bVIj6Wl_-etP3yarQhxQxtNRdXInjtpvhHR38MpbobUPUDa59LUvEJgcdot5yBbc8eIna5hjkbfJBQttjQn3hfIkj0DKLtD8bMC-RDjt35n-Wqxobbtof-KOhLTrJaDkWsx7Oy4oTj6DD5lrG0P6RHmb8ht59JROPSU7mhqb_3MvB-fnEbf7r-2TP_R6GBPQtqMbIQft20-DIeMtjBMJaJRCqWR7jWhk2hl72ybCMQlRX5q79atTMfNTJ-qcH0KQpsIJM5-DWbT8EjHCet5DJJn4j_Dv5b-0aKRcY-tT5M-Lf5eT22-usy6Qd2hcH0KLKDh6gb4PhQKuZ5qutLTb4QTbqWKJcKfb1MRjvMPnF-tKZDb-JXtr92nuDal5TtUthSDnTDMRhXfIL04nyKMnitnr9-pnLJpQrh459XP68bTkA5bjZKxtq3mkjbPbDfn02eCKuj6tWj6j0DNRabK6aKC5bL6rJabC3b5CzXU6q2bDeQN3OW4Rq3Irt2M8aQI0WjJ3oyU7k0q0vWtvJWbbvLT7johRTWqR4enjb3MonDh83Mxb4BUrCHRrzWn3O5hvvhKoO3MA-yUKmDloOW-TB5bbPLUQF5l8-sq0x0bOte-bQXH_E5bj2qRCqVIKa3f',\n",
    "'BDSFRCVID_BFESS': '8--OJexroG0xMovDbuOS5T78igKKHJQTDYLtOwXPsp3LGJLVgaSTEG0PtjcEHMA-2ZlgogKK02OTH6KF_2uxOjjg8UtVJeC6EG0Ptf8g0M5',\n",
    "'H_BDCLCKID_SF_BFESS': 'tJPqoKtbtDI3fP36qR3KhPt8Kpby2D62aKDs2nopBhcqEIL4QTQM5p5yQ2c7LUvtynT2KJnz3Po8MUbSj4QoDjFjXJ7RJRJbK6vwKJ5s5h5nhMJSb67JDMP0-4F8exry523ioIovQpn0MhQ3DRoWXPIqbN7P-p5Z5mAqKl0MLPbtbb0xXj_0D6bBjHujtT_s2TTKLPK8fCnBDP59MDTjhPrMypomWMT-0bFH_-5L-l5js56SbU5hW5LSQxQ3QhLDQNn7_JjOX-0bVIj6Wl_-etP3yarQhxQxtNRdXInjtpvhHR38MpbobUPUDa59LUvEJgcdot5yBbc8eIna5hjkbfJBQttjQn3hfIkj0DKLtD8bMC-RDjt35n-Wqxobbtof-KOhLTrJaDkWsx7Oy4oTj6DD5lrG0P6RHmb8ht59JROPSU7mhqb_3MvB-fnEbf7r-2TP_R6GBPQtqMbIQft20-DIeMtjBMJaJRCqWR7jWhk2hl72ybCMQlRX5q79atTMfNTJ-qcH0KQpsIJM5-DWbT8EjHCet5DJJn4j_Dv5b-0aKRcY-tT5M-Lf5eT22-usy6Qd2hcH0KLKDh6gb4PhQKuZ5qutLTb4QTbqWKJcKfb1MRjvMPnF-tKZDb-JXtr92nuDal5TtUthSDnTDMRhXfIL04nyKMnitnr9-pnLJpQrh459XP68bTkA5bjZKxtq3mkjbPbDfn02eCKuj6tWj6j0DNRabK6aKC5bL6rJabC3b5CzXU6q2bDeQN3OW4Rq3Irt2M8aQI0WjJ3oyU7k0q0vWtvJWbbvLT7johRTWqR4enjb3MonDh83Mxb4BUrCHRrzWn3O5hvvhKoO3MA-yUKmDloOW-TB5bbPLUQF5l8-sq0x0bOte-bQXH_E5bj2qRCqVIKa3f',\n",
    "'indexPageSugList': '%5B%22%E7%8B%97%E7%8B%97%22%5D',\n",
    "'cleanHistoryStatus': '0',\n",
    "'BAIDUID_BFESS': '104BD58A7C408DABABCAC9E0A1B184B4:FG=1',\n",
    "'BDRCVFR[dG2JNJb_ajR]': 'mk3SLVN4HKm',\n",
    "'BDRCVFR[-pGxjrCMryR]': 'mk3SLVN4HKm',\n",
    "'ab_sr': '1.0.1_Y2YxZDkwMWZkMmY2MzA4MGU0OTNhMzVlNTcwMmM2MWE4YWU4OTc1ZjZmZDM2N2RjYmVkMzFiY2NjNWM4Nzk4NzBlZTliYWU0ZTAyODkzNDA3YzNiMTVjMTllMzQ0MGJlZjAwYzk5MDdjNWM0MzJmMDdhOWNhYTZhMjIwODc5MDMxN2QyMmE1YTFmN2QyY2M1M2VmZDkzMjMyOThiYmNhZA==',\n",
    "'delPer': '0',\n",
    "'PSINO': '2',\n",
    "'BA_HECTOR': '8h24a024042g05alup1h3g0aq0q',\n",
    "}\n",
    "\n",
    "headers = {\n",
    "'Connection': 'keep-alive',\n",
    "'sec-ch-ua': '\" Not;A Brand\";v=\"99\", \"Google Chrome\";v=\"97\", \"Chromium\";v=\"97\"',\n",
    "'Accept': 'text/plain, */*; q=0.01',\n",
    "'X-Requested-With': 'XMLHttpRequest',\n",
    "'sec-ch-ua-mobile': '?0',\n",
    "'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36',\n",
    "'sec-ch-ua-platform': '\"macOS\"',\n",
    "'Sec-Fetch-Site': 'same-origin',\n",
    "'Sec-Fetch-Mode': 'cors',\n",
    "'Sec-Fetch-Dest': 'empty',\n",
    "'Referer': 'https://image.baidu.com/search/index?tn=baiduimage&ipn=r&ct=201326592&cl=2&lm=-1&st=-1&fm=result&fr=&sf=1&fmq=1647837998851_R&pv=&ic=&nc=1&z=&hd=&latest=&copyright=&se=1&showtab=0&fb=0&width=&height=&face=0&istype=2&dyTabStr=MCwzLDIsNiwxLDUsNCw4LDcsOQ%3D%3D&ie=utf-8&sid=&word=%E7%8B%97%E7%8B%97',\n",
    "'Accept-Language': 'zh-CN,zh;q=0.9',\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "60a4f342",
   "metadata": {},
   "source": [
    "# 主程序"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f56bca6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def craw_single_class(keyword, DOWNLOAD_NUM = 200):\n",
    "    if os.path.exists('dataset/'+keyword):\n",
    "        print('文件夹 dataset/{} 已存在，之后直接将爬取到的图片保存至该文件夹中'.format(keyword))\n",
    "    else:\n",
    "        os.makedirs('dataset/{}'.format(keyword))\n",
    "        print('新建文件夹：dataset/{}'.format(keyword))\n",
    "    count = 1\n",
    "    \n",
    "    with tqdm(total=DOWNLOAD_NUM, position=0, leave=True) as pbar:\n",
    "        \n",
    "        # 爬取第几张\n",
    "        num = 0\n",
    "\n",
    "        # 是否继续爬取\n",
    "        FLAG = True\n",
    "\n",
    "        while FLAG:\n",
    "\n",
    "            page = 30 * count\n",
    "\n",
    "            params = (\n",
    "                ('tn', 'resultjson_com'),\n",
    "                ('logid', '12508239107856075440'),\n",
    "                ('ipn', 'rj'),\n",
    "                ('ct', '201326592'),\n",
    "                ('is', ''),\n",
    "                ('fp', 'result'),\n",
    "                ('fr', ''),\n",
    "                ('word', f'{keyword}'),\n",
    "                ('queryWord', f'{keyword}'),\n",
    "                ('cl', '2'),\n",
    "                ('lm', '-1'),\n",
    "                ('ie', 'utf-8'),\n",
    "                ('oe', 'utf-8'),\n",
    "                ('adpicid', ''),\n",
    "                ('st', '-1'),\n",
    "                ('z', ''),\n",
    "                ('ic', ''),\n",
    "                ('hd', ''),\n",
    "                ('latest', ''),\n",
    "                ('copyright', ''),\n",
    "                ('s', ''),\n",
    "                ('se', ''),\n",
    "                ('tab', ''),\n",
    "                ('width', ''),\n",
    "                ('height', ''),\n",
    "                ('face', '0'),\n",
    "                ('istype', '2'),\n",
    "                ('qc', ''),\n",
    "                ('nc', '1'),\n",
    "                ('expermode', ''),\n",
    "                ('nojc', ''),\n",
    "                ('isAsync', ''),\n",
    "                ('pn', f'{page}'),\n",
    "                ('rn', '30'),\n",
    "                ('gsm', '1e'),\n",
    "                ('1647838001666', ''),\n",
    "            )\n",
    "\n",
    "            response = requests.get('https://image.baidu.com/search/acjson', headers=headers, params=params, cookies=cookies)\n",
    "            if response.status_code == 200:\n",
    "                try:\n",
    "                    json_data = response.json().get(\"data\")\n",
    "\n",
    "                    if json_data:\n",
    "                        for x in json_data:\n",
    "                            type = x.get(\"type\")\n",
    "                            if type not in [\"gif\"]:\n",
    "                                img = x.get(\"thumbURL\")\n",
    "                                fromPageTitleEnc = x.get(\"fromPageTitleEnc\")\n",
    "                                try:\n",
    "                                    resp = requests.get(url=img, verify=False)\n",
    "                                    time.sleep(1)\n",
    "                                    # print(f\"链接 {img}\")\n",
    "\n",
    "                                    # 保存文件名\n",
    "                                    # file_save_path = f'dataset/{keyword}/{num}-{fromPageTitleEnc}.{type}'\n",
    "                                    file_save_path = f'dataset/{keyword}/{num}.{type}'\n",
    "                                    with open(file_save_path, 'wb') as f:                                    \n",
    "                                        f.write(resp.content)\n",
    "                                        f.flush()\n",
    "                                        # print('第 {} 张图像 {} 爬取完成'.format(num, fromPageTitleEnc))\n",
    "                                        num += 1\n",
    "                                        pbar.update(1) # 进度条更新\n",
    "\n",
    "                                    # 爬取数量达到要求\n",
    "                                    if num > DOWNLOAD_NUM:\n",
    "                                        FLAG = False\n",
    "                                        print('{} 张图像爬取完毕'.format(num))\n",
    "                                        break\n",
    "\n",
    "                                except Exception:\n",
    "                                    pass\n",
    "                except:\n",
    "                    pass\n",
    "            else:\n",
    "                break\n",
    "\n",
    "            count += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5614a74",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "81\n"
     ]
    }
   ],
   "source": [
    "class_list = ['人参果','佛手瓜','哈密瓜','圣女果','山楂','山竹','无花果','木瓜','李子','杏',\n",
    "              '杨桃','杨梅','枇杷','枣','柚子','柠檬','柿子','树莓','桂圆','桑葚',\n",
    "              '梨','椰子','榴莲','樱桃','橘子','毛丹','水蜜桃','沃柑','沙果','沙棘','油桃',\n",
    "              '牛油果','猕猴桃','甘蔗','甜瓜-伊丽莎白','甜瓜-白','甜瓜-绿','甜瓜-金','番石榴-百',\n",
    "              '番石榴-红','白兰瓜','白心火龙果','白萝卜','百香果','石榴','砂糖橘','粑粑柑','红心火龙果',\n",
    "              '红苹果','羊奶果','羊角蜜','胡萝卜','脐橙','腰果','芒果','芦柑','草莓','荔枝','莲雾','菠萝',\n",
    "              '菠萝莓','菠萝蜜','葡萄-白','葡萄-红','蓝莓','蛇皮果','蟠桃','血橙','西柚','西梅','西瓜',\n",
    "              '西红柿','车厘子','酸角','金桔','青柠','青苹果','香橼','香蕉','黄桃','黑莓']\n",
    "print(len(class_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d70f1979",
   "metadata": {},
   "outputs": [],
   "source": [
    "for each in class_list:\n",
    "    craw_single_class(each, DOWNLOAD_NUM = 200)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "cv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
